# Output File Analysis:
# the file called freqOfTermsInQueriesExtended.txt contains 38871 unique query terms with their frequency in the query trace
# the meaning here for the queryTermFreq in queries is that: how many queries have contained the specific query term.
# NOTE: if this query contains multiple times of this specific query term, we just count 1 time.(Major differences compared to column2)
# So the output file freqOfTermsInQueriesExtended.txt will have 3 columns compared to 2 columns. 
# can be regarded as one machine learned feature

# Program inputs:
# (1) inputFileName1 = ""

# Program outputs:
# (1) outputFileName = ""


from struct import *
import sys

print "Begin:"
# step1: load the query Terms ONLY, not loading the freq 
queryTermFreqDict = {}
inputFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/freqOfTermsInQueries_sortedByQueryTerm.txt"
inputFileHandler1 = open(inputFileName1,"r")
for line in inputFileHandler1.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    if queryTerm not in queryTermFreqDict:
        queryTermFreqDict[queryTerm] = 0
    else:
        pass
    
print "len(queryTermFreqDict):",len(queryTermFreqDict)
inputFileHandler1.close()

# option1:
# inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueries"
# option2:
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueries_head_95K"
inputFileHandler = open(inputFileName,"r")

for line in inputFileHandler.readlines():
    currentQueryTermFreqDict = {}
    # print "line:",line.strip()
    queryTermList = line.strip().split(":")[1].strip().split(" ")
    # print "queryTermList:",queryTermList
    
    data = ""
    for element in queryTermList:
        data += element + " "
    
    # print "data(old):",data
    # print "original data:",data
    
    for i in range(0,len(data)):
        # print "data[i]:",ord(data[i])
        if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
            # Just replace them with a space.
            data = data[:i] + " " + data[i+1:]

    # print "data(new):",data
    
    newQueryTermList = data.strip().split(" ")
    
    for queryTerm in newQueryTermList:
        if queryTerm.strip() != "":
            queryTermLower = queryTerm.lower()
            if queryTermLower not in currentQueryTermFreqDict:
                currentQueryTermFreqDict[queryTermLower] = 0
            else:
                pass
    
    # Let's do a comparizon between this dict and the overall dict in order to updated the over dict :)
    for term in currentQueryTermFreqDict:
        if term in queryTermFreqDict:
            queryTermFreqDict[term] += 1
        else:
            print "Unexpected Behavior"
            print "mark1"
            exit(1)
    
inputFileHandler.close()


outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/freqOfTermsIn100KQueries_head_95K"
outputFileHandler = open(outputFileName,"w")

for queryTerm in queryTermFreqDict:
    outputFileHandler.write(queryTerm + " " + str(queryTermFreqDict[queryTerm]) + "\n")

outputFileHandler.close()

print "End..."

'''
print "Begin:"
# the input files should have 2
# one for queries that are human judged, and one for queries that are NOT just for effeciency task for the year 2006
# please add the special char filer for all our project

queryTermFreqDict = {}

inputFileNameList = []

inputFileName1 = "/home/obukai/workspace/polyIRToolkit/polyIRIndexer/04-06.topics.701-850.polyIRTKCompatibleMode"
inputFileName2 = "/home/obukai/workspace/polyIRToolkit/polyIRIndexer/06.efficiency_topics.all"

# This file is used just for testing the filter to normalize the query terms, the only thing is to be consistent with what the polyIRToolkit is originally doing.
# inputFileName3 = "/home/obukai/workspace/polyIRToolkit/polyIRIndexer/difficultQueries.test"

# current output path
outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/freqOfTermsInQueries.txt"

# old output path
# outputFileName = "/data5/team/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/freqOfTermsInQueries.txt"
outputFileHandler = open(outputFileName,"w")


inputFileNameList.append(inputFileName1)
inputFileNameList.append(inputFileName2)
#inputFileNameList.append(inputFileName3)


for inputFileName in inputFileNameList:
    inputFileHandler = open(inputFileName,"r")
    
    for line in inputFileHandler.readlines():
        # print "line:",line.strip()
        queryTermList = line.strip().split(":")[1].strip().split(" ")
        # print "queryTermList:",queryTermList
        
        data = ""
        for element in queryTermList:
            data += element + " "
        
        # print "data(old):",data
        # print "original data:",data
        
        for i in range(0,len(data)):
            # print "data[i]:",ord(data[i])
            if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
                # Just replace them with a space.
                data = data[:i] + " " + data[i+1:]
    
        # print "data(new):",data
        
        newQueryTermList = data.strip().split(" ")
        
        for queryTerm in newQueryTermList:
            if queryTerm.strip() != "":
                queryTermLower = queryTerm.lower()
                if queryTermLower not in queryTermFreqDict:
                    queryTermFreqDict[queryTermLower] = 1
                else:
                    queryTermFreqDict[queryTermLower] += 1


            
# print "queryTermFreqDict:",queryTermFreqDict
overallQueryTermList = queryTermFreqDict.keys()
overallQueryTermList.sort()

for queryTerm in overallQueryTermList:
    outputFileHandler.write(queryTerm + " " + str( queryTermFreqDict[queryTerm] ) + "\n")
    print queryTerm,queryTermFreqDict[queryTerm]


inputFileHandler.close()
outputFileHandler.close()
print "End..."
'''
